import pandas as pd
pd.set_option('display.expand_frame_repr', False)
import matplotlib
import matplotlib.pyplot as plt
font = {'size' : 20}
matplotlib.rc('font', **font)
import seaborn as sns
from pylab import rcParams
rcParams["figure.figsize"] = 30,16
from collections import OrderedDict
import datetime as dt
from datetime import date
from datetime import datetime
import sklearn
from sklearn import metrics
from sklearn.model_selection import KFold
from sklearn.preprocessing import MinMaxScaler
import joblib
import numpy as np
np.random.seed(0)
import scipy
import datetime as dt
from datetime import date
import tensorflow
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, LSTM, Activation, InputLayer
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.regularizers import l2
import warnings
warnings.filterwarnings("ignore")
import sys
sys.path.insert(0, "../")
import functions
#Funktion für RMSE erstellen
from keras import backend as K
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
Das Netz lässt sich durch eine weitere LSTM-Schicht vergrößern. Dafür wird zunächst zur ersten Schicht aus 32 Neuronen eine weitere Schicht mit 64 Neuronen hinzugefügt, um mehr Potenzial für Generalisierungen zu bieten.
#Hyperparameter
epochs = 100
batch_size = 8
window_size = 14
def model_creation():
#Modell erstellen
model = Sequential()
model.add(InputLayer(input_shape=(window_size, 4)))
model.add(LSTM(units=32, return_sequences=True, activation="tanh"))
model.add(LSTM(units=64, return_sequences=False, activation="tanh"))
model.add(Dense(units=1))
adam = Adam(learning_rate=0.0005)
model.compile(loss=root_mean_squared_error, optimizer=adam)
return model
#Unskalierte Daten für Analysen laden
df_unscaled = pd.read_csv("../3-Data Preparation/data.csv", index_col=0, parse_dates=True)
df_unscaled.index.freq = "D"
y_test_true = df_unscaled["verbrauch"]["2021-01-01":]
y_train_true = df_unscaled["verbrauch"][window_size:2192]
#Skalierte Daten für Modellierung laden
df_scaled = pd.read_csv("../3-Data Preparation/data_scaled.csv", index_col=0, parse_dates=True)
df_scaled.index.freq = "D"
#Aufteilung X (Merkmale) und y (Ziel)
X = df_scaled[["verbrauch","arbeitstag","temperatur","tagesstunden"]]
#Stromverbrauch wird bei X um eine Stelle nach vorne verschoben, daher entfällt der 01.01.2015
X["arbeitstag"] = X["arbeitstag"].shift(-1)
X["temperatur"] = X["temperatur"].shift(-1)
X["tagesstunden"] = X["tagesstunden"].shift(-1)
X = X[:2556]
y = df_scaled["verbrauch"]
#Aufteilung der Daten in Zeitfenster
def restructure_data(px, py, window_size):
X_, y_ = [], []
idx_range = range(len(px) - (window_size) + 1)
for idx in idx_range:
X_.append(px[idx:idx+window_size])
y_.append(py[idx+window_size])
X_ = np.array(X_)
y_ = np.array(y_)
return X_, y_
X_windows, y_windows = restructure_data(X, y, window_size)
#Aufteilung in Trainings-, Validierungs- und Testdaten
split_by = 2557 - 365 - window_size
X_train = X_windows[:split_by]
y_train = y_windows[:split_by]
X_test = X_windows[split_by:]
y_test = y_windows[split_by:]
#Kreuzvalidierung erstellen
kfold = KFold(n_splits=5, shuffle=True, random_state=0)
#DataFrames für History und Metriken
df_history = pd.DataFrame()
df_history.index.name = "Epoch"
df_metrics = pd.DataFrame(columns=["Testdaten", "Trainingsdaten", "Validierungsdaten"])
df_metrics.index.name = "Split"
iteration = 1
scaler_target = joblib.load("../3-Data Preparation/scaler_endog.save")
#Kreuzvalidierung
for train_index, validation_index in kfold.split(X_train, y_train):
print(iteration, "-", end="\t")
#Modell erstellen
model = model_creation()
#Modell trainieren
history = model.fit(x=X_train[train_index], y=y_train[train_index], epochs=epochs, batch_size=batch_size, validation_data=(X_train[validation_index], y_train[validation_index]), verbose=0)
#Ergebnisse speichern
df_history[str(iteration) + "_train_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["loss"]).reshape(-1, 1)) - 102469)).squeeze()
df_history[str(iteration) + "_validation_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["val_loss"]).reshape(-1, 1)) - 102469)).squeeze()
#Vorhersagen für Test-, Trainings- und Validierungsdaten
preds_test = scaler_target.inverse_transform(model.predict(X_test).reshape(-1, 1))
preds_train = scaler_target.inverse_transform(model.predict(X_train[train_index]).reshape(-1, 1))
preds_validation = scaler_target.inverse_transform(model.predict(X_train[validation_index]).reshape(-1, 1))
df_metrics.loc[iteration] = [round(metrics.mean_absolute_percentage_error(y_test_true, preds_test) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[train_index], preds_train) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[validation_index], preds_validation) * 100, 2)]
iteration = iteration + 1
#Durchschnittswerte bei History und Metriken berechnen
df_history["train_loss"] = (df_history["1_train_loss"] + df_history["2_train_loss"] + df_history["3_train_loss"] + df_history["4_train_loss"] + df_history["5_train_loss"]) / 5
df_history["validation_loss"] = (df_history["1_validation_loss"] + df_history["2_validation_loss"] + df_history["3_validation_loss"] + df_history["4_validation_loss"] + df_history["5_validation_loss"]) / 5
df_metrics.loc["Average"] = [round(df_metrics["Testdaten"].mean(), 2),
round(df_metrics["Trainingsdaten"].mean(), 2),
round(df_metrics["Validierungsdaten"].mean(), 2)]
1 - 2 - 3 - 4 - 5 -
#Training auswerten (Metriken und Lernkurve)
functions.evaluate_training(df_metrics, df_history, 25000)
Testdaten Trainingsdaten Validierungsdaten Split 1 2.35 1.50 1.70 2 2.54 1.55 1.85 3 2.05 1.57 1.80 4 2.60 1.59 1.80 5 2.17 1.54 1.71 Average 2.34 1.55 1.77
Die Lernkurve deutet wie schon beim einschichtigen LSTM auf eine zu hohe Lernrate hin. Die Konvergenz auf den Testdaten setzt in etwa nach 70 Epochen ein, ab dann führt weiteres Training zu Überanpassungen.
Das Modell hat sich im Vergleich zum einschichtigen LSTM-Netz nicht verbessert. Allerdings werden die Trainingsdaten besser erkannt, was auf eine Überanpassung hindeutet.
#Finales Modell erstellen
model = model_creation()
#Finales Modell trainieren
history = model.fit(x=X_train, y=y_train, epochs=epochs, batch_size=batch_size, use_multiprocessing=True)
Epoch 1/100 273/273 [==============================] - 5s 9ms/step - loss: 0.1927 Epoch 2/100 273/273 [==============================] - 2s 8ms/step - loss: 0.1172 Epoch 3/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0700 Epoch 4/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0562 Epoch 5/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0539 Epoch 6/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0504 Epoch 7/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0472 Epoch 8/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0455 Epoch 9/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0453 Epoch 10/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0426 Epoch 11/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0412 Epoch 12/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0414 Epoch 13/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0387 Epoch 14/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0393 Epoch 15/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0378 Epoch 16/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0376 Epoch 17/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0366 Epoch 18/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0364 Epoch 19/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0350 Epoch 20/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0369 Epoch 21/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0361 Epoch 22/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0349 Epoch 23/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0339 Epoch 24/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0346 Epoch 25/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0335 Epoch 26/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0338 Epoch 27/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0333 Epoch 28/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0335 Epoch 29/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0327 Epoch 30/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0344 Epoch 31/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0345 Epoch 32/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0334 Epoch 33/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0328 Epoch 34/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0330 Epoch 35/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0328 Epoch 36/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0326 Epoch 37/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0315 Epoch 38/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0316 Epoch 39/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0311 Epoch 40/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0315 Epoch 41/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0311 Epoch 42/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0311 Epoch 43/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0309 Epoch 44/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0313 Epoch 45/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0326 Epoch 46/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0308 Epoch 47/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0306 Epoch 48/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0297 Epoch 49/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0300 Epoch 50/100 273/273 [==============================] - 2s 9ms/step - loss: 0.0301 Epoch 51/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0310 Epoch 52/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0306 Epoch 53/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0298 Epoch 54/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0291 Epoch 55/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0297 Epoch 56/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0288 Epoch 57/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0286 Epoch 58/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0286 Epoch 59/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0293 Epoch 60/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0289 Epoch 61/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0281 Epoch 62/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0279 Epoch 63/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0273 Epoch 64/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0276 Epoch 65/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0279 Epoch 66/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0271 Epoch 67/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0268 Epoch 68/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0267 Epoch 69/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0265 Epoch 70/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0270 Epoch 71/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0262 Epoch 72/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0265 Epoch 73/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0266 Epoch 74/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0270 Epoch 75/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0259 Epoch 76/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0262 Epoch 77/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0268 Epoch 78/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0260 Epoch 79/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0254 Epoch 80/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0259 Epoch 81/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0250 Epoch 82/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0251 Epoch 83/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0252 Epoch 84/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0250 Epoch 85/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0249 Epoch 86/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0250 Epoch 87/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0247 Epoch 88/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0244 Epoch 89/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0245 Epoch 90/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0247 Epoch 91/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0245 Epoch 92/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0243 Epoch 93/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0243 Epoch 94/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0241 Epoch 95/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0235 Epoch 96/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0244 Epoch 97/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0242 Epoch 98/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0237 Epoch 99/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0240 Epoch 100/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0232
#Vorhersagen erzeugen
scaled_preds_test = model.predict(X_test)
scaled_preds_train = model.predict(X_train)
functions.custom_metrics_lstm(y_test_true, scaled_preds_test, y_train_true, scaled_preds_train)
Testdaten Trainingsdaten
R2 0.96 0.98
MAE 4139.0 2592.0
MSE 30218619.0 12450978.0
RMSE 5497.0 3529.0
MAPE 2.48 % 1.59 %
In diesem Schritt wird das LSTM-Netz durch mehr Neuronen in der ersten Schicht erweitert.
#Hyperparameter
epochs = 100
batch_size = 8
window_size = 14
def model_creation():
#Modell erstellen
model = Sequential()
model.add(InputLayer(input_shape=(window_size, 4)))
model.add(LSTM(units=64, return_sequences=True, activation="tanh"))
model.add(LSTM(units=64, return_sequences=False, activation="tanh"))
model.add(Dense(units=1))
adam = Adam(learning_rate=0.0005)
model.compile(loss=root_mean_squared_error, optimizer=adam)
return model
#Unskalierte Daten für Analysen laden
df_unscaled = pd.read_csv("../3-Data Preparation/data.csv", index_col=0, parse_dates=True)
df_unscaled.index.freq = "D"
y_test_true = df_unscaled["verbrauch"]["2021-01-01":]
y_train_true = df_unscaled["verbrauch"][window_size:2192]
#Skalierte Daten für Modellierung laden
df_scaled = pd.read_csv("../3-Data Preparation/data_scaled.csv", index_col=0, parse_dates=True)
df_scaled.index.freq = "D"
#Aufteilung X (Merkmale) und y (Ziel)
X = df_scaled[["verbrauch","arbeitstag","temperatur","tagesstunden"]]
#Stromverbrauch wird bei X um eine Stelle nach vorne verschoben, daher entfällt der 01.01.2015
X["arbeitstag"] = X["arbeitstag"].shift(-1)
X["temperatur"] = X["temperatur"].shift(-1)
X["tagesstunden"] = X["tagesstunden"].shift(-1)
X = X[:2556]
y = df_scaled["verbrauch"]
#Aufteilung der Daten in Zeitfenster
def restructure_data(px, py, window_size):
X_, y_ = [], []
idx_range = range(len(px) - (window_size) + 1)
for idx in idx_range:
X_.append(px[idx:idx+window_size])
y_.append(py[idx+window_size])
X_ = np.array(X_)
y_ = np.array(y_)
return X_, y_
X_windows, y_windows = restructure_data(X, y, window_size)
#Aufteilung in Trainings-, Validierungs- und Testdaten
split_by = 2557 - 365 - window_size
X_train = X_windows[:split_by]
y_train = y_windows[:split_by]
X_test = X_windows[split_by:]
y_test = y_windows[split_by:]
#Kreuzvalidierung erstellen
kfold = KFold(n_splits=5, shuffle=True, random_state=0)
#DataFrames für History und Metriken
df_history = pd.DataFrame()
df_history.index.name = "Epoch"
df_metrics = pd.DataFrame(columns=["Testdaten", "Trainingsdaten", "Validierungsdaten"])
df_metrics.index.name = "Split"
iteration = 1
scaler_target = joblib.load("../3-Data Preparation/scaler_endog.save")
#Kreuzvalidierung
for train_index, validation_index in kfold.split(X_train, y_train):
print(iteration, "-", end="\t")
#Modell erstellen
model = model_creation()
#Modell trainieren
history = model.fit(x=X_train[train_index], y=y_train[train_index], epochs=epochs, batch_size=batch_size, validation_data=(X_train[validation_index], y_train[validation_index]), verbose=0)
#Ergebnisse speichern
df_history[str(iteration) + "_train_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["loss"]).reshape(-1, 1)) - 102469)).squeeze()
df_history[str(iteration) + "_validation_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["val_loss"]).reshape(-1, 1)) - 102469)).squeeze()
#Vorhersagen für Test-, Trainings- und Validierungsdaten
preds_test = scaler_target.inverse_transform(model.predict(X_test).reshape(-1, 1))
preds_train = scaler_target.inverse_transform(model.predict(X_train[train_index]).reshape(-1, 1))
preds_validation = scaler_target.inverse_transform(model.predict(X_train[validation_index]).reshape(-1, 1))
df_metrics.loc[iteration] = [round(metrics.mean_absolute_percentage_error(y_test_true, preds_test) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[train_index], preds_train) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[validation_index], preds_validation) * 100, 2)]
iteration = iteration + 1
#Durchschnittswerte bei History und Metriken berechnen
df_history["train_loss"] = (df_history["1_train_loss"] + df_history["2_train_loss"] + df_history["3_train_loss"] + df_history["4_train_loss"] + df_history["5_train_loss"]) / 5
df_history["validation_loss"] = (df_history["1_validation_loss"] + df_history["2_validation_loss"] + df_history["3_validation_loss"] + df_history["4_validation_loss"] + df_history["5_validation_loss"]) / 5
df_metrics.loc["Average"] = [round(df_metrics["Testdaten"].mean(), 2),
round(df_metrics["Trainingsdaten"].mean(), 2),
round(df_metrics["Validierungsdaten"].mean(), 2)]
1 - 2 - 3 - 4 - 5 -
#Training auswerten (Metriken und Lernkurve)
functions.evaluate_training(df_metrics, df_history, 25000)
Testdaten Trainingsdaten Validierungsdaten Split 1 2.21 1.66 1.81 2 2.24 1.40 1.62 3 2.59 1.44 1.70 4 2.95 1.63 1.95 5 2.24 1.47 1.59 Average 2.45 1.52 1.73
Die Lernkurve deutet wie schon beim einschichtigen LSTM auf eine zu hohe Lernrate hin. Die Konvergenz auf den Testdaten setzt in etwa nach 70 Epochen ein, ab dann führt weiteres Training zu Überanpassungen.
Die zusätzlichen Neuronen bringen keinen weiteren Nutzen. Die Metriken verbessern sich kaum, die Testdaten werden während der Kreuzvalidierung sogar schlechter erkannt. Daher ist von weiteren Vergrößerungen abzusehen. Stattdessen sollte noch einmal die Lernrate optimiert werden.
#Finales Modell erstellen
model = model_creation()
#Finales Modell trainieren
history = model.fit(x=X_train, y=y_train, epochs=epochs, batch_size=batch_size, use_multiprocessing=True)
Epoch 1/100 273/273 [==============================] - 5s 8ms/step - loss: 0.1743 Epoch 2/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0816 Epoch 3/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0592 Epoch 4/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0544 Epoch 5/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0497 Epoch 6/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0471 Epoch 7/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0442 Epoch 8/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0431 Epoch 9/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0426 Epoch 10/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0399 Epoch 11/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0412 Epoch 12/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0398 Epoch 13/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0379 Epoch 14/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0367 Epoch 15/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0361 Epoch 16/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0364 Epoch 17/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0352 Epoch 18/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0346 Epoch 19/100 273/273 [==============================] - 2s 9ms/step - loss: 0.0347 Epoch 20/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0346 Epoch 21/100 273/273 [==============================] - 3s 9ms/step - loss: 0.0348 Epoch 22/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0346 Epoch 23/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0342 Epoch 24/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0332 Epoch 25/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0327 Epoch 26/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0325 Epoch 27/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0329 Epoch 28/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0322 Epoch 29/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0333 Epoch 30/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0319 Epoch 31/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0318 Epoch 32/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0316 Epoch 33/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0316 Epoch 34/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0315 Epoch 35/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0313 Epoch 36/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0309 Epoch 37/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0313 Epoch 38/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0310 Epoch 39/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0301 Epoch 40/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0303 Epoch 41/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0310 Epoch 42/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0297 Epoch 43/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0297 Epoch 44/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0301 Epoch 45/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0294 Epoch 46/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0294 Epoch 47/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0294 Epoch 48/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0294 Epoch 49/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0290 Epoch 50/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0292 Epoch 51/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0283 Epoch 52/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0281 Epoch 53/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0276 Epoch 54/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0282 Epoch 55/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0277 Epoch 56/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0285 Epoch 57/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0277 Epoch 58/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0267 Epoch 59/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0270 Epoch 60/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0274 Epoch 61/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0273 Epoch 62/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0274 Epoch 63/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0281 Epoch 64/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0273 Epoch 65/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0269 Epoch 66/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0271 Epoch 67/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0266 Epoch 68/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0262 Epoch 69/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0265 Epoch 70/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0268 Epoch 71/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0261 Epoch 72/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0260 Epoch 73/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0265 Epoch 74/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0262 Epoch 75/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0264 Epoch 76/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0263 Epoch 77/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0259 Epoch 78/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0253 Epoch 79/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0260 Epoch 80/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0254 Epoch 81/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0254 Epoch 82/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0253 Epoch 83/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0253 Epoch 84/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0254 Epoch 85/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0253 Epoch 86/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0254 Epoch 87/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0254 Epoch 88/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0248 Epoch 89/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0247 Epoch 90/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0252 Epoch 91/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0252 Epoch 92/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0252 Epoch 93/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0255 Epoch 94/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0247 Epoch 95/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0240 Epoch 96/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0245 Epoch 97/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0248 Epoch 98/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0245 Epoch 99/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0246 Epoch 100/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0244
#Vorhersagen erzeugen
scaled_preds_test = model.predict(X_test)
scaled_preds_train = model.predict(X_train)
functions.custom_metrics_lstm(y_test_true, scaled_preds_test, y_train_true, scaled_preds_train)
Testdaten Trainingsdaten
R2 0.96 0.98
MAE 3905.0 2449.0
MSE 25934455.0 11549078.0
RMSE 5093.0 3398.0
MAPE 2.34 % 1.5 %
Da der Verlauf der Lernkurve immer noch durch eine zu hohe Lernrate begründet sein könnte, wird nun statt 0,0005 eine niedrigere Rate von 0,0001 verwendet.
#Hyperparameter
epochs = 100
batch_size = 8
window_size = 14
def model_creation():
#Modell erstellen
model = Sequential()
model.add(InputLayer(input_shape=(window_size, 4)))
model.add(LSTM(units=32, return_sequences=True, activation="tanh"))
model.add(LSTM(units=64, return_sequences=False, activation="tanh"))
model.add(Dense(units=1))
adam = Adam(learning_rate=0.0001)
model.compile(loss=root_mean_squared_error, optimizer=adam)
return model
#Unskalierte Daten für Analysen laden
df_unscaled = pd.read_csv("../3-Data Preparation/data.csv", index_col=0, parse_dates=True)
df_unscaled.index.freq = "D"
y_test_true = df_unscaled["verbrauch"]["2021-01-01":]
y_train_true = df_unscaled["verbrauch"][window_size:2192]
#Skalierte Daten für Modellierung laden
df_scaled = pd.read_csv("../3-Data Preparation/data_scaled.csv", index_col=0, parse_dates=True)
df_scaled.index.freq = "D"
#Aufteilung X (Merkmale) und y (Ziel)
X = df_scaled[["verbrauch","arbeitstag","temperatur","tagesstunden"]]
#Stromverbrauch wird bei X um eine Stelle nach vorne verschoben, daher entfällt der 01.01.2015
X["arbeitstag"] = X["arbeitstag"].shift(-1)
X["temperatur"] = X["temperatur"].shift(-1)
X["tagesstunden"] = X["tagesstunden"].shift(-1)
X = X[:2556]
y = df_scaled["verbrauch"]
#Aufteilung der Daten in Zeitfenster
def restructure_data(px, py, window_size):
X_, y_ = [], []
idx_range = range(len(px) - (window_size) + 1)
for idx in idx_range:
X_.append(px[idx:idx+window_size])
y_.append(py[idx+window_size])
X_ = np.array(X_)
y_ = np.array(y_)
return X_, y_
X_windows, y_windows = restructure_data(X, y, window_size)
#Aufteilung in Trainings-, Validierungs- und Testdaten
split_by = 2557 - 365 - window_size
X_train = X_windows[:split_by]
y_train = y_windows[:split_by]
X_test = X_windows[split_by:]
y_test = y_windows[split_by:]
#Kreuzvalidierung erstellen
kfold = KFold(n_splits=5, shuffle=True, random_state=0)
#DataFrames für History und Metriken
df_history = pd.DataFrame()
df_history.index.name = "Epoch"
df_metrics = pd.DataFrame(columns=["Testdaten", "Trainingsdaten", "Validierungsdaten"])
df_metrics.index.name = "Split"
iteration = 1
scaler_target = joblib.load("../3-Data Preparation/scaler_endog.save")
#Kreuzvalidierung
for train_index, validation_index in kfold.split(X_train, y_train):
print(iteration, "-", end="\t")
#Modell erstellen
model = model_creation()
#Modell trainieren
history = model.fit(x=X_train[train_index], y=y_train[train_index], epochs=epochs, batch_size=batch_size, validation_data=(X_train[validation_index], y_train[validation_index]), verbose=0)
#Ergebnisse speichern
df_history[str(iteration) + "_train_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["loss"]).reshape(-1, 1)) - 102469)).squeeze()
df_history[str(iteration) + "_validation_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["val_loss"]).reshape(-1, 1)) - 102469)).squeeze()
#Vorhersagen für Test-, Trainings- und Validierungsdaten
preds_test = scaler_target.inverse_transform(model.predict(X_test).reshape(-1, 1))
preds_train = scaler_target.inverse_transform(model.predict(X_train[train_index]).reshape(-1, 1))
preds_validation = scaler_target.inverse_transform(model.predict(X_train[validation_index]).reshape(-1, 1))
df_metrics.loc[iteration] = [round(metrics.mean_absolute_percentage_error(y_test_true, preds_test) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[train_index], preds_train) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[validation_index], preds_validation) * 100, 2)]
iteration = iteration + 1
#Durchschnittswerte bei History und Metriken berechnen
df_history["train_loss"] = (df_history["1_train_loss"] + df_history["2_train_loss"] + df_history["3_train_loss"] + df_history["4_train_loss"] + df_history["5_train_loss"]) / 5
df_history["validation_loss"] = (df_history["1_validation_loss"] + df_history["2_validation_loss"] + df_history["3_validation_loss"] + df_history["4_validation_loss"] + df_history["5_validation_loss"]) / 5
df_metrics.loc["Average"] = [round(df_metrics["Testdaten"].mean(), 2),
round(df_metrics["Trainingsdaten"].mean(), 2),
round(df_metrics["Validierungsdaten"].mean(), 2)]
1 - 2 - 3 - 4 - 5 -
#Training auswerten (Metriken und Lernkurve)
functions.evaluate_training(df_metrics, df_history, 25000)
Testdaten Trainingsdaten Validierungsdaten Split 1 2.53 2.12 2.18 2 2.53 1.90 1.90 3 2.32 2.04 2.21 4 2.68 1.96 2.20 5 2.50 2.16 2.09 Average 2.51 2.04 2.12
Die Lernkurve verläuft allerdings trotzdem fast gleich, nur dass sich das Modell etwas zu verschlechtern scheint.
An den Metriken ist zu erkennen, dass das Modell wie auch schon bei vorherigen Modellen bei einer Lernrate von 0,0001 zu Unteranpassungen neigt. Da die Lernkurve allerdings nach etwa 100 Epochen mit der Konvergenz beginnt, ist davon auszugehen, dass weitere Epochen das Modell nicht mehr ausreichend verbessern.
#Finales Modell erstellen
model = model_creation()
#Finales Modell trainieren
history = model.fit(x=X_train, y=y_train, epochs=epochs, batch_size=batch_size, use_multiprocessing=True)
Epoch 1/100 273/273 [==============================] - 5s 8ms/step - loss: 0.2060 Epoch 2/100 273/273 [==============================] - 2s 8ms/step - loss: 0.1785 Epoch 3/100 273/273 [==============================] - 2s 8ms/step - loss: 0.1610 Epoch 4/100 273/273 [==============================] - 2s 8ms/step - loss: 0.1379 Epoch 5/100 273/273 [==============================] - 2s 8ms/step - loss: 0.1047 Epoch 6/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0808 Epoch 7/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0742 Epoch 8/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0676 Epoch 9/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0646 Epoch 10/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0614 Epoch 11/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0581 Epoch 12/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0577 Epoch 13/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0557 Epoch 14/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0534 Epoch 15/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0538 Epoch 16/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0525 Epoch 17/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0521 Epoch 18/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0516 Epoch 19/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0509 Epoch 20/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0523 Epoch 21/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0485 Epoch 22/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0481 Epoch 23/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0485 Epoch 24/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0472 Epoch 25/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0469 Epoch 26/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0483 Epoch 27/100 273/273 [==============================] - 2s 9ms/step - loss: 0.0464 Epoch 28/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0476 Epoch 29/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0466 Epoch 30/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0448 Epoch 31/100 273/273 [==============================] - 2s 9ms/step - loss: 0.0445 Epoch 32/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0455 Epoch 33/100 273/273 [==============================] - 2s 9ms/step - loss: 0.0452 Epoch 34/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0451 Epoch 35/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0436 Epoch 36/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0433 Epoch 37/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0443 Epoch 38/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0431 Epoch 39/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0430 Epoch 40/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0423 Epoch 41/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0429 Epoch 42/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0426 Epoch 43/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0423 Epoch 44/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0420 Epoch 45/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0420 Epoch 46/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0415 Epoch 47/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0409 Epoch 48/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0410 Epoch 49/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0412 Epoch 50/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0411 Epoch 51/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0414 Epoch 52/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0395 Epoch 53/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0398 Epoch 54/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0396 Epoch 55/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0397 Epoch 56/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0391 Epoch 57/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0390 Epoch 58/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0392 Epoch 59/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0390 Epoch 60/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0385 Epoch 61/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0378 Epoch 62/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0373 Epoch 63/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0378 Epoch 64/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0374 Epoch 65/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0368 Epoch 66/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0376 Epoch 67/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0372 Epoch 68/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0373 Epoch 69/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0361 Epoch 70/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0364 Epoch 71/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0372 Epoch 72/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0363 Epoch 73/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0362 Epoch 74/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0356 Epoch 75/100 273/273 [==============================] - 2s 9ms/step - loss: 0.0353 Epoch 76/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0351 Epoch 77/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0350 Epoch 78/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0345 Epoch 79/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0345 Epoch 80/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0348 Epoch 81/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0346 Epoch 82/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0341 Epoch 83/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0343 Epoch 84/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0335 Epoch 85/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0334 Epoch 86/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0333 Epoch 87/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0335 Epoch 88/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0342 Epoch 89/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0336 Epoch 90/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0335 Epoch 91/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0328 Epoch 92/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0334 Epoch 93/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0327 Epoch 94/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0336 Epoch 95/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0329 Epoch 96/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0327 Epoch 97/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0331 Epoch 98/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0324 Epoch 99/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0323 Epoch 100/100 273/273 [==============================] - 2s 8ms/step - loss: 0.0326
#Vorhersagen erzeugen
scaled_preds_test = model.predict(X_test)
scaled_preds_train = model.predict(X_train)
functions.custom_metrics_lstm(y_test_true, scaled_preds_test, y_train_true, scaled_preds_train)
Testdaten Trainingsdaten
R2 0.95 0.97
MAE 4305.0 3014.0
MSE 33273755.0 19953733.0
RMSE 5768.0 4467.0
MAPE 2.64 % 1.9 %
Da das Netz durch die zwei LSTM-Schichten und deren ausgerollten Zeitschichten mittlerweile etwas tiefer geworden ist, wird zu Sicherheit noch einmal überprüft, in wie weit sich das Modell mittels ReLu verbessern lässt.
#Hyperparameter
epochs = 100
batch_size = 8
window_size = 14
def model_creation():
#Modell erstellen
model = Sequential()
model.add(InputLayer(input_shape=(window_size, 4)))
model.add(LSTM(units=32, return_sequences=True, activation="relu"))
model.add(LSTM(units=64, return_sequences=False, activation="relu"))
model.add(Dense(units=1))
adam = Adam(learning_rate=0.0005)
model.compile(loss=root_mean_squared_error, optimizer=adam)
return model
#Unskalierte Daten für Analysen laden
df_unscaled = pd.read_csv("../3-Data Preparation/data.csv", index_col=0, parse_dates=True)
df_unscaled.index.freq = "D"
y_test_true = df_unscaled["verbrauch"]["2021-01-01":]
y_train_true = df_unscaled["verbrauch"][window_size:2192]
#Skalierte Daten für Modellierung laden
df_scaled = pd.read_csv("../3-Data Preparation/data_scaled.csv", index_col=0, parse_dates=True)
df_scaled.index.freq = "D"
#Aufteilung X (Merkmale) und y (Ziel)
X = df_scaled[["verbrauch","arbeitstag","temperatur","tagesstunden"]]
#Stromverbrauch wird bei X um eine Stelle nach vorne verschoben, daher entfällt der 01.01.2015
X["arbeitstag"] = X["arbeitstag"].shift(-1)
X["temperatur"] = X["temperatur"].shift(-1)
X["tagesstunden"] = X["tagesstunden"].shift(-1)
X = X[:2556]
y = df_scaled["verbrauch"]
#Aufteilung der Daten in Zeitfenster
def restructure_data(px, py, window_size):
X_, y_ = [], []
idx_range = range(len(px) - (window_size) + 1)
for idx in idx_range:
X_.append(px[idx:idx+window_size])
y_.append(py[idx+window_size])
X_ = np.array(X_)
y_ = np.array(y_)
return X_, y_
X_windows, y_windows = restructure_data(X, y, window_size)
#Aufteilung in Trainings-, Validierungs- und Testdaten
split_by = 2557 - 365 - window_size
X_train = X_windows[:split_by]
y_train = y_windows[:split_by]
X_test = X_windows[split_by:]
y_test = y_windows[split_by:]
#Kreuzvalidierung erstellen
kfold = KFold(n_splits=5, shuffle=True, random_state=0)
#DataFrames für History und Metriken
df_history = pd.DataFrame()
df_history.index.name = "Epoch"
df_metrics = pd.DataFrame(columns=["Testdaten", "Trainingsdaten", "Validierungsdaten"])
df_metrics.index.name = "Split"
iteration = 1
scaler_target = joblib.load("../3-Data Preparation/scaler_endog.save")
#Kreuzvalidierung
for train_index, validation_index in kfold.split(X_train, y_train):
print(iteration, "-", end="\t")
#Modell erstellen
model = model_creation()
#Modell trainieren
history = model.fit(x=X_train[train_index], y=y_train[train_index], epochs=epochs, batch_size=batch_size, validation_data=(X_train[validation_index], y_train[validation_index]), verbose=0)
#Ergebnisse speichern
df_history[str(iteration) + "_train_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["loss"]).reshape(-1, 1)) - 102469)).squeeze()
df_history[str(iteration) + "_validation_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["val_loss"]).reshape(-1, 1)) - 102469)).squeeze()
#Vorhersagen für Test-, Trainings- und Validierungsdaten
preds_test = scaler_target.inverse_transform(model.predict(X_test).reshape(-1, 1))
preds_train = scaler_target.inverse_transform(model.predict(X_train[train_index]).reshape(-1, 1))
preds_validation = scaler_target.inverse_transform(model.predict(X_train[validation_index]).reshape(-1, 1))
df_metrics.loc[iteration] = [round(metrics.mean_absolute_percentage_error(y_test_true, preds_test) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[train_index], preds_train) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[validation_index], preds_validation) * 100, 2)]
iteration = iteration + 1
#Durchschnittswerte bei History und Metriken berechnen
df_history["train_loss"] = (df_history["1_train_loss"] + df_history["2_train_loss"] + df_history["3_train_loss"] + df_history["4_train_loss"] + df_history["5_train_loss"]) / 5
df_history["validation_loss"] = (df_history["1_validation_loss"] + df_history["2_validation_loss"] + df_history["3_validation_loss"] + df_history["4_validation_loss"] + df_history["5_validation_loss"]) / 5
df_metrics.loc["Average"] = [round(df_metrics["Testdaten"].mean(), 2),
round(df_metrics["Trainingsdaten"].mean(), 2),
round(df_metrics["Validierungsdaten"].mean(), 2)]
1 - 2 - 3 - 4 - 5 -
#Training auswerten (Metriken und Lernkurve)
functions.evaluate_training(df_metrics, df_history, 25000)
Testdaten Trainingsdaten Validierungsdaten Split 1 2.04 1.40 1.61 2 2.73 1.74 1.87 3 1.96 1.45 1.65 4 2.84 1.75 2.03 5 2.09 1.60 1.75 Average 2.33 1.59 1.78
Die ReLu führt allerdings zu einer Überanpassung an die Trainingsdaten. Auf den Trainingsdaten lässt sich der MAPE senken, allerdings steigt er auf den Testdaten leicht an.
#Finales Modell erstellen
model = model_creation()
#Finales Modell trainieren
history = model.fit(x=X_train, y=y_train, epochs=epochs, batch_size=batch_size, use_multiprocessing=True)
Epoch 1/100 273/273 [==============================] - 3s 6ms/step - loss: 0.2104 Epoch 2/100 273/273 [==============================] - 2s 6ms/step - loss: 0.1582 Epoch 3/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0752 Epoch 4/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0594 Epoch 5/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0539 Epoch 6/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0499 Epoch 7/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0491 Epoch 8/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0454 Epoch 9/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0440 Epoch 10/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0436 Epoch 11/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0417 Epoch 12/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0422 Epoch 13/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0403 Epoch 14/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0390 Epoch 15/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0391 Epoch 16/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0388 Epoch 17/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0387 Epoch 18/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0389 Epoch 19/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0370 Epoch 20/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0365 Epoch 21/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0363 Epoch 22/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0356 Epoch 23/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0343 Epoch 24/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0336 Epoch 25/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0340 Epoch 26/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0345 Epoch 27/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0340 Epoch 28/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0332 Epoch 29/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0332 Epoch 30/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0325 Epoch 31/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0331 Epoch 32/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0332 Epoch 33/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0314 Epoch 34/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0312 Epoch 35/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0311 Epoch 36/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0312 Epoch 37/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0318 Epoch 38/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0309 Epoch 39/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0305 Epoch 40/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0305 Epoch 41/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0305 Epoch 42/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0301 Epoch 43/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0308 Epoch 44/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0300 Epoch 45/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0302 Epoch 46/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0290 Epoch 47/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0291 Epoch 48/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0284 Epoch 49/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0285 Epoch 50/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0284 Epoch 51/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0288 Epoch 52/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0281 Epoch 53/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0277 Epoch 54/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0280 Epoch 55/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0280 Epoch 56/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0276 Epoch 57/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0290 Epoch 58/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0282 Epoch 59/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0269 Epoch 60/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0266 Epoch 61/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0274 Epoch 62/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0274 Epoch 63/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0269 Epoch 64/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0266 Epoch 65/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0267 Epoch 66/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0270 Epoch 67/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0266 Epoch 68/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0258 Epoch 69/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0264 Epoch 70/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0260 Epoch 71/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0256 Epoch 72/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0260 Epoch 73/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0261 Epoch 74/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0257 Epoch 75/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0252 Epoch 76/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0263 Epoch 77/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0259 Epoch 78/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0252 Epoch 79/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0250 Epoch 80/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0252 Epoch 81/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0253 Epoch 82/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0256 Epoch 83/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0254 Epoch 84/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0253 Epoch 85/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0252 Epoch 86/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0248 Epoch 87/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0245 Epoch 88/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0257 Epoch 89/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0246 Epoch 90/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0245 Epoch 91/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0248 Epoch 92/100 273/273 [==============================] - 2s 7ms/step - loss: 0.0245 Epoch 93/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0243 Epoch 94/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0244 Epoch 95/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0244 Epoch 96/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0243 Epoch 97/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0244 Epoch 98/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0243 Epoch 99/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0244 Epoch 100/100 273/273 [==============================] - 2s 6ms/step - loss: 0.0242
#Vorhersagen erzeugen
scaled_preds_test = model.predict(X_test)
scaled_preds_train = model.predict(X_train)
functions.custom_metrics_lstm(y_test_true, scaled_preds_test, y_train_true, scaled_preds_train)
Testdaten Trainingsdaten
R2 0.95 0.98
MAE 4525.0 2909.0
MSE 32740926.0 14199486.0
RMSE 5722.0 3768.0
MAPE 2.69 % 1.74 %
Um sicherzustellen, dass die Verlustfunktion nicht in einem lokalen Minimum hängen bleibt, wird der Training noch einmal mit größeren Batches durchgeführt.
#Hyperparameter
epochs = 100
batch_size = 16
window_size = 14
def model_creation():
#Modell erstellen
model = Sequential()
model.add(InputLayer(input_shape=(window_size, 4)))
model.add(LSTM(units=32, return_sequences=True, activation="tanh"))
model.add(LSTM(units=64, return_sequences=False, activation="tanh"))
model.add(Dense(units=1))
adam = Adam(learning_rate=0.0005)
model.compile(loss=root_mean_squared_error, optimizer=adam)
return model
#Unskalierte Daten für Analysen laden
df_unscaled = pd.read_csv("../3-Data Preparation/data.csv", index_col=0, parse_dates=True)
df_unscaled.index.freq = "D"
y_test_true = df_unscaled["verbrauch"]["2021-01-01":]
y_train_true = df_unscaled["verbrauch"][window_size:2192]
#Skalierte Daten für Modellierung laden
df_scaled = pd.read_csv("../3-Data Preparation/data_scaled.csv", index_col=0, parse_dates=True)
df_scaled.index.freq = "D"
#Aufteilung X (Merkmale) und y (Ziel)
X = df_scaled[["verbrauch","arbeitstag","temperatur","tagesstunden"]]
#Stromverbrauch wird bei X um eine Stelle nach vorne verschoben, daher entfällt der 01.01.2015
X["arbeitstag"] = X["arbeitstag"].shift(-1)
X["temperatur"] = X["temperatur"].shift(-1)
X["tagesstunden"] = X["tagesstunden"].shift(-1)
X = X[:2556]
y = df_scaled["verbrauch"]
#Aufteilung der Daten in Zeitfenster
def restructure_data(px, py, window_size):
X_, y_ = [], []
idx_range = range(len(px) - (window_size) + 1)
for idx in idx_range:
X_.append(px[idx:idx+window_size])
y_.append(py[idx+window_size])
X_ = np.array(X_)
y_ = np.array(y_)
return X_, y_
X_windows, y_windows = restructure_data(X, y, window_size)
#Aufteilung in Trainings-, Validierungs- und Testdaten
split_by = 2557 - 365 - window_size
X_train = X_windows[:split_by]
y_train = y_windows[:split_by]
X_test = X_windows[split_by:]
y_test = y_windows[split_by:]
#Kreuzvalidierung erstellen
kfold = KFold(n_splits=5, shuffle=True, random_state=0)
#DataFrames für History und Metriken
df_history = pd.DataFrame()
df_history.index.name = "Epoch"
df_metrics = pd.DataFrame(columns=["Testdaten", "Trainingsdaten", "Validierungsdaten"])
df_metrics.index.name = "Split"
iteration = 1
scaler_target = joblib.load("../3-Data Preparation/scaler_endog.save")
#Kreuzvalidierung
for train_index, validation_index in kfold.split(X_train, y_train):
print(iteration, "-", end="\t")
#Modell erstellen
model = model_creation()
#Modell trainieren
history = model.fit(x=X_train[train_index], y=y_train[train_index], epochs=epochs, batch_size=batch_size, validation_data=(X_train[validation_index], y_train[validation_index]), verbose=0)
#Ergebnisse speichern
df_history[str(iteration) + "_train_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["loss"]).reshape(-1, 1)) - 102469)).squeeze()
df_history[str(iteration) + "_validation_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["val_loss"]).reshape(-1, 1)) - 102469)).squeeze()
#Vorhersagen für Test-, Trainings- und Validierungsdaten
preds_test = scaler_target.inverse_transform(model.predict(X_test).reshape(-1, 1))
preds_train = scaler_target.inverse_transform(model.predict(X_train[train_index]).reshape(-1, 1))
preds_validation = scaler_target.inverse_transform(model.predict(X_train[validation_index]).reshape(-1, 1))
df_metrics.loc[iteration] = [round(metrics.mean_absolute_percentage_error(y_test_true, preds_test) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[train_index], preds_train) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[validation_index], preds_validation) * 100, 2)]
iteration = iteration + 1
#Durchschnittswerte bei History und Metriken berechnen
df_history["train_loss"] = (df_history["1_train_loss"] + df_history["2_train_loss"] + df_history["3_train_loss"] + df_history["4_train_loss"] + df_history["5_train_loss"]) / 5
df_history["validation_loss"] = (df_history["1_validation_loss"] + df_history["2_validation_loss"] + df_history["3_validation_loss"] + df_history["4_validation_loss"] + df_history["5_validation_loss"]) / 5
df_metrics.loc["Average"] = [round(df_metrics["Testdaten"].mean(), 2),
round(df_metrics["Trainingsdaten"].mean(), 2),
round(df_metrics["Validierungsdaten"].mean(), 2)]
1 - 2 - 3 - 4 - 5 -
#Training auswerten (Metriken und Lernkurve)
functions.evaluate_training(df_metrics, df_history, 25000)
Testdaten Trainingsdaten Validierungsdaten Split 1 2.35 1.64 1.80 2 2.59 1.65 1.78 3 2.57 1.78 1.83 4 2.57 1.76 2.02 5 2.21 1.64 1.69 Average 2.46 1.69 1.82
Die größeren Batches führen allerdings zu einem Abfall der Qualität der Vorhersagen während der Kreuzvalidierung. Beim finalen Test schneidet das Modell in etwa gleich ab, eine Verbesserung ist hierdurch aber nicht zu erzielen. Viel kleiner sollten die Batches allerdings auch nicht gemacht werden, da das Training an ansonsten sehr lange dauert und nicht davon auszugehen ist, dass die Verlustfunktion das ihr Minimum dadurch besser erreichen kann. Die Batchgröße ist also angemessen.
#Finales Modell erstellen
model = model_creation()
#Finales Modell trainieren
history = model.fit(x=X_train, y=y_train, epochs=epochs, batch_size=batch_size, use_multiprocessing=True)
Epoch 1/100 137/137 [==============================] - 4s 8ms/step - loss: 0.2038 Epoch 2/100 137/137 [==============================] - 1s 8ms/step - loss: 0.1643 Epoch 3/100 137/137 [==============================] - 1s 8ms/step - loss: 0.1030 Epoch 4/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0745 Epoch 5/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0623 Epoch 6/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0559 Epoch 7/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0521 Epoch 8/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0508 Epoch 9/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0487 Epoch 10/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0466 Epoch 11/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0471 Epoch 12/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0461 Epoch 13/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0449 Epoch 14/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0431 Epoch 15/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0420 Epoch 16/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0410 Epoch 17/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0423 Epoch 18/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0427 Epoch 19/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0405 Epoch 20/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0411 Epoch 21/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0388 Epoch 22/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0401 Epoch 23/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0403 Epoch 24/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0390 Epoch 25/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0385 Epoch 26/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0369 Epoch 27/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0384 Epoch 28/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0365 Epoch 29/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0374 Epoch 30/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0358 Epoch 31/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0356 Epoch 32/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0367 Epoch 33/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0365 Epoch 34/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0357 Epoch 35/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0359 Epoch 36/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0362 Epoch 37/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0348 Epoch 38/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0352 Epoch 39/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0338 Epoch 40/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0359 Epoch 41/100 137/137 [==============================] - 1s 10ms/step - loss: 0.0361 Epoch 42/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0341 Epoch 43/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0338 Epoch 44/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0335 Epoch 45/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0338 Epoch 46/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0337 Epoch 47/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0340 Epoch 48/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0333 Epoch 49/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0334 Epoch 50/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0334 Epoch 51/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0323 Epoch 52/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0332 Epoch 53/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0321 Epoch 54/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0327 Epoch 55/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0326 Epoch 56/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0325 Epoch 57/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0328 Epoch 58/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0318 Epoch 59/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0329 Epoch 60/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0331 Epoch 61/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0320 Epoch 62/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0332 Epoch 63/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0322 Epoch 64/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0319 Epoch 65/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0315 Epoch 66/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0326 Epoch 67/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0315 Epoch 68/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0310 Epoch 69/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0313 Epoch 70/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0322 Epoch 71/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0305 Epoch 72/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0307 Epoch 73/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0306 Epoch 74/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0314 Epoch 75/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0304 Epoch 76/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0309 Epoch 77/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0300 Epoch 78/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0302 Epoch 79/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0299 Epoch 80/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0300 Epoch 81/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0300 Epoch 82/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0304 Epoch 83/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0298 Epoch 84/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0296 Epoch 85/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0294 Epoch 86/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0294 Epoch 87/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0296 Epoch 88/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0290 Epoch 89/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0288 Epoch 90/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0290 Epoch 91/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0296 Epoch 92/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0289 Epoch 93/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0286 Epoch 94/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0283 Epoch 95/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0282 Epoch 96/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0277 Epoch 97/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0278 Epoch 98/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0283 Epoch 99/100 137/137 [==============================] - 1s 8ms/step - loss: 0.0291 Epoch 100/100 137/137 [==============================] - 1s 9ms/step - loss: 0.0281
#Vorhersagen erzeugen
scaled_preds_test = model.predict(X_test)
scaled_preds_train = model.predict(X_train)
functions.custom_metrics_lstm(y_test_true, scaled_preds_test, y_train_true, scaled_preds_train)
Testdaten Trainingsdaten
R2 0.96 0.98
MAE 4214.0 2878.0
MSE 31126323.0 15189041.0
RMSE 5579.0 3897.0
MAPE 2.58 % 1.77 %
Die bisher probierten Architekturen und Trainingsparameter führen nicht zu ausreichenden Ergebnissen. Letztlich soll allerdings noch ein Netz mit mehr Neuronen ausprobiert werden. Dafür werden 128 Neuronen in beiden Schichten eingefügt.
#Hyperparameter
epochs = 100
batch_size = 8
window_size = 14
def model_creation():
#Modell erstellen
model = Sequential()
model.add(InputLayer(input_shape=(window_size, 4)))
model.add(LSTM(units=128, return_sequences=True, activation="tanh"))
model.add(LSTM(units=128, return_sequences=False, activation="tanh"))
model.add(Dense(units=1))
adam = Adam(learning_rate=0.0005)
model.compile(loss=root_mean_squared_error, optimizer=adam)
return model
#Unskalierte Daten für Analysen laden
df_unscaled = pd.read_csv("../3-Data Preparation/data.csv", index_col=0, parse_dates=True)
df_unscaled.index.freq = "D"
y_test_true = df_unscaled["verbrauch"]["2021-01-01":]
y_train_true = df_unscaled["verbrauch"][window_size:2192]
#Skalierte Daten für Modellierung laden
df_scaled = pd.read_csv("../3-Data Preparation/data_scaled.csv", index_col=0, parse_dates=True)
df_scaled.index.freq = "D"
#Aufteilung X (Merkmale) und y (Ziel)
X = df_scaled[["verbrauch","arbeitstag","temperatur","tagesstunden"]]
#Stromverbrauch wird bei X um eine Stelle nach vorne verschoben, daher entfällt der 01.01.2015
X["arbeitstag"] = X["arbeitstag"].shift(-1)
X["temperatur"] = X["temperatur"].shift(-1)
X["tagesstunden"] = X["tagesstunden"].shift(-1)
X = X[:2556]
y = df_scaled["verbrauch"]
#Aufteilung der Daten in Zeitfenster
def restructure_data(px, py, window_size):
X_, y_ = [], []
idx_range = range(len(px) - (window_size) + 1)
for idx in idx_range:
X_.append(px[idx:idx+window_size])
y_.append(py[idx+window_size])
X_ = np.array(X_)
y_ = np.array(y_)
return X_, y_
X_windows, y_windows = restructure_data(X, y, window_size)
#Aufteilung in Trainings-, Validierungs- und Testdaten
split_by = 2557 - 365 - window_size
X_train = X_windows[:split_by]
y_train = y_windows[:split_by]
X_test = X_windows[split_by:]
y_test = y_windows[split_by:]
#Kreuzvalidierung erstellen
kfold = KFold(n_splits=5, shuffle=True, random_state=0)
#DataFrames für History und Metriken
df_history = pd.DataFrame()
df_history.index.name = "Epoch"
df_metrics = pd.DataFrame(columns=["Testdaten", "Trainingsdaten", "Validierungsdaten"])
df_metrics.index.name = "Split"
iteration = 1
scaler_target = joblib.load("../3-Data Preparation/scaler_endog.save")
#Kreuzvalidierung
for train_index, validation_index in kfold.split(X_train, y_train):
print(iteration, "-", end="\t")
#Modell erstellen
model = model_creation()
#Modell trainieren
history = model.fit(x=X_train[train_index], y=y_train[train_index], epochs=epochs, batch_size=batch_size, validation_data=(X_train[validation_index], y_train[validation_index]), verbose=0)
#Ergebnisse speichern
df_history[str(iteration) + "_train_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["loss"]).reshape(-1, 1)) - 102469)).squeeze()
df_history[str(iteration) + "_validation_loss"] = pd.DataFrame(data=(scaler_target.inverse_transform(np.array(history.history["val_loss"]).reshape(-1, 1)) - 102469)).squeeze()
#Vorhersagen für Test-, Trainings- und Validierungsdaten
preds_test = scaler_target.inverse_transform(model.predict(X_test).reshape(-1, 1))
preds_train = scaler_target.inverse_transform(model.predict(X_train[train_index]).reshape(-1, 1))
preds_validation = scaler_target.inverse_transform(model.predict(X_train[validation_index]).reshape(-1, 1))
df_metrics.loc[iteration] = [round(metrics.mean_absolute_percentage_error(y_test_true, preds_test) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[train_index], preds_train) * 100, 2),
round(metrics.mean_absolute_percentage_error(y_train_true[validation_index], preds_validation) * 100, 2)]
iteration = iteration + 1
#Durchschnittswerte bei History und Metriken berechnen
df_history["train_loss"] = (df_history["1_train_loss"] + df_history["2_train_loss"] + df_history["3_train_loss"] + df_history["4_train_loss"] + df_history["5_train_loss"]) / 5
df_history["validation_loss"] = (df_history["1_validation_loss"] + df_history["2_validation_loss"] + df_history["3_validation_loss"] + df_history["4_validation_loss"] + df_history["5_validation_loss"]) / 5
df_metrics.loc["Average"] = [round(df_metrics["Testdaten"].mean(), 2),
round(df_metrics["Trainingsdaten"].mean(), 2),
round(df_metrics["Validierungsdaten"].mean(), 2)]
1 - 2 - 3 - 4 - 5 -
#Training auswerten (Metriken und Lernkurve)
functions.evaluate_training(df_metrics, df_history, 25000)
Testdaten Trainingsdaten Validierungsdaten Split 1 2.15 1.35 1.56 2 2.91 1.69 1.87 3 2.53 1.44 1.70 4 2.32 1.38 1.74 5 1.99 1.40 1.60 Average 2.38 1.45 1.69
Der MAPE wird nicht verbessert (im Vergleich zu LSTM-LSTM-2), obwohl es sich um ein sehr viel umfangreicheres Netz handelt. Es scheint, dass das Potenzial selbst mit kleinen Netzen sehr schnell erschlossen werden kann, mehr Neuronen, Schichten oder ein längeres Training tragen nur wenig zu Verbesserungen bei.
#Finales Modell erstellen
model = model_creation()
#Finales Modell trainieren
history = model.fit(x=X_train, y=y_train, epochs=epochs, batch_size=batch_size, use_multiprocessing=True)
Epoch 1/100 273/273 [==============================] - 6s 13ms/step - loss: 0.1525 Epoch 2/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0619 Epoch 3/100 273/273 [==============================] - 4s 14ms/step - loss: 0.0537 Epoch 4/100 273/273 [==============================] - 4s 14ms/step - loss: 0.0474 Epoch 5/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0457 Epoch 6/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0429 Epoch 7/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0413 Epoch 8/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0401 Epoch 9/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0402 Epoch 10/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0383 Epoch 11/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0361 Epoch 12/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0361 Epoch 13/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0361 Epoch 14/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0353 Epoch 15/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0350 Epoch 16/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0347 Epoch 17/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0353 Epoch 18/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0334 Epoch 19/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0333 Epoch 20/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0329 Epoch 21/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0334 Epoch 22/100 273/273 [==============================] - 4s 14ms/step - loss: 0.0327 Epoch 23/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0329 Epoch 24/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0348 Epoch 25/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0324 Epoch 26/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0323 Epoch 27/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0330 Epoch 28/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0312 Epoch 29/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0315 Epoch 30/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0316 Epoch 31/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0311 Epoch 32/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0315 Epoch 33/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0317 Epoch 34/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0302 Epoch 35/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0301 Epoch 36/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0301 Epoch 37/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0302 Epoch 38/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0295 Epoch 39/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0300 Epoch 40/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0296 Epoch 41/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0291 Epoch 42/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0288 Epoch 43/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0292 Epoch 44/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0283 Epoch 45/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0283 Epoch 46/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0282 Epoch 47/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0287 Epoch 48/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0280 Epoch 49/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0277 Epoch 50/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0283 Epoch 51/100 273/273 [==============================] - 4s 14ms/step - loss: 0.0278 Epoch 52/100 273/273 [==============================] - 4s 14ms/step - loss: 0.0266 Epoch 53/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0274 Epoch 54/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0273 Epoch 55/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0263 Epoch 56/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0265 Epoch 57/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0264 Epoch 58/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0271 Epoch 59/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0263 Epoch 60/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0268 Epoch 61/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0267 Epoch 62/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0268 Epoch 63/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0269 Epoch 64/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0263 Epoch 65/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0256 Epoch 66/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0253 Epoch 67/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0256 Epoch 68/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0258 Epoch 69/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0256 Epoch 70/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0255 Epoch 71/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0250 Epoch 72/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0251 Epoch 73/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0245 Epoch 74/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0250 Epoch 75/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0254 Epoch 76/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0244 Epoch 77/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0241 Epoch 78/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0244 Epoch 79/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0243 Epoch 80/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0246 Epoch 81/100 273/273 [==============================] - 3s 12ms/step - loss: 0.0240 Epoch 82/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0240 Epoch 83/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0245 Epoch 84/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0239 Epoch 85/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0242 Epoch 86/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0241 Epoch 87/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0241 Epoch 88/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0236 Epoch 89/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0239 Epoch 90/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0236 Epoch 91/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0237 Epoch 92/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0235 Epoch 93/100 273/273 [==============================] - 3s 13ms/step - loss: 0.0235 Epoch 94/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0236 Epoch 95/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0236 Epoch 96/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0232 Epoch 97/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0236 Epoch 98/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0237 Epoch 99/100 273/273 [==============================] - 4s 13ms/step - loss: 0.0232 Epoch 100/100 273/273 [==============================] - 4s 14ms/step - loss: 0.0239
#Vorhersagen erzeugen
scaled_preds_test = model.predict(X_test)
scaled_preds_train = model.predict(X_train)
functions.custom_metrics_lstm(y_test_true, scaled_preds_test, y_train_true, scaled_preds_train)
Testdaten Trainingsdaten
R2 0.96 0.98
MAE 3904.0 2370.0
MSE 27930955.0 10918003.0
RMSE 5285.0 3304.0
MAPE 2.36 % 1.47 %
